#coding:utf-8
'''
Created on 2010-12-20

@author: fengbingjian
'''
import sys,re,urllib,traceback,types,time,threading
from exeSql import exeSql

class catch:
    
    url = "http://news.sina.com.cn/"
    topRegex = "<a href=\"(http://(ent|tech|games|slide)\.sina\.com\.cn/[^<>]+/\d+\.shtml)\" target=(\"|)_blank(\"|)>([^<>]+)</a>"
    subRegex = "<!-- google_ad_section_start -->(.*)<!-- google_ad_section_end -->"
    subContentRegex = "<p.*?>([^<>]+)</p>"
    topUrlId=0
    topTitleId=4
    mark = 0
    topList = None
    sql = None
    config = []
    config_id = 0
    avi_id = 0
    chart_set = "gbk"
    news_list=[]
    show = None
    
    def __init__(self , show = None):
        
        
        
        print u"初始化..."
        self.add_config(
                      {"urlBase": "http://bbs.meizu.com/",
                       'url':"http://bbs.meizu.com/forumdisplay.php?fid=22",
                      "topRegex" : "<a href=\"(thread-\d+-\d+-\d+.html)\" title=\"([^\"]+)\">[^<]+</a>",
                      "subRegex" : "<div id=\"postmessage_\d+\" class=\"t_msgfont\">([^div]+)</div>",
                      "subContentRegex" : None,
                      "topUrlId" : 0,
                      "topTitleId" : 1,
                      "max" : 10,"subContentList":-1,
                      "chart_set":'gbk'})
        self.add_config(
                        {"urlBase": "",
                          "url":"http://news.sina.com.cn/", 
                          "topRegex" : "<a href=\"(http://(news|tech|games)\.sina\.com\.cn/[^<>]+/\d+\.shtml)\" target=(\"|)_blank(\"|)>([^<>]+)</a>",
                          "subRegex" : "<!-- google_ad_section_start -->(.*)<!-- google_ad_section_end -->",
                          "subContentRegex" : "<p.*?>([^<>]+)</p>",
                          "topUrlId" : 0,
                          "topTitleId" : 4,
                          "max" : 5,
                          "subContentList" : 0,
                          "chart_set":'gbk'})
        
        self.add_config(
                        {"urlBase": "http://bj.58.com/hezu/",
                          "url":"http://bj.58.com/hezu/", 
                          "topRegex" : '<a href="([^"]+)" target="_blank" class="t">([^<]+)</a>',
                          "subRegex" : "<div class=\"conleft\">(.*)<ul class=\"imgs\">",
                          "subContentRegex" : None,
                          "topUrlId" : 0,
                          "topTitleId" : 1,
                          "max" : 10,
                          "subContentList" : 0,
                          "chart_set":'utf-8'})
      
        #self.start()
        self.sql = exeSql()
        self.news_list = []
        threading._start_new_thread(self.catch, (self.news_list , None))
        if show != None:
            self.show = show
        #self.catch(self.news_list, None)
    def add_config(self , config):
        self.config.append(config)      
    def get_src(self,url):
        if url != None:
            print u'抓取url:' + url
            wp = urllib.urlopen(url)
            return wp.read()#.decode(self.chart_set)
        else:
            print u"url不能为空"
            
    def get_list_by_regex(self,content , regex , flags=0):
        if content != None and regex != None:
            return re.compile(regex, flags).findall(content)
        else:
            print u"content或者regex为空"
            
#    def start(self):
#        self.mark = 0
#        self.config_id=0
#        self.topList=None
#        self.avi_id = 0
        #self.topList = self.get_list_by_regex(self.get_src(self.url) , self.topRegex , re.M|re.S)
        #print u"共" + str(len(self.topList)) + u"条新闻"
    
    def catch(self , list , t):
        i = 1
        while True:
            print i 
            i += 1
            for c in self.config:
                try:
                    self.topList =self.get_list_by_regex(self.get_src(c['url']) , c['topRegex'] , re.M|re.S)
                except Exception , e:
                    print e
                    self.topList = None
                avi_id = 0
                if self.topList != None and len(self.topList) > 0:
                    for line in self.topList:
                            try:
                                target_url = line[c['topUrlId']]
                                
                                if target_url.startswith("http") == False:
                                    target_url = c["urlBase"] + "/"+target_url
                                #print "target_url=" + target_url
                                if self.sql.if_exist(target_url):
                                    print u"不是新数据"
                                    #return "","",""
                                else:
                                    #print self.mark,self.topList[self.mark][0]
                                    print u"新数据"
                                    content =  self.get_list_by_regex(self.get_src(target_url) , c['subRegex'] , re.M|re.S)
                                    #print len(content),type(content)
                                    
                                    #print type(content)
                                    #print len(content)
                                    
                                    
                                    if content != None and len(content) > 0:
                                        subContentListId = c["subContentList"]
                                        if subContentListId == -1:
                                            temp_list =[]
                                            i = 0
                                            
                                            for l in content:
                                                if i == 0:
                                                    temp_list.append(u"楼主说：")
                                                    temp_list.append(l.decode(self.chart_set))
                                                else:
                                                    temp_list.append("\n")
                                                    temp_list.append(str(i))
                                                    temp_list.append(u"楼说:")
                                                    temp_list.append(l.decode(self.chart_set))
                                                i = i+1
                                            content = "".join(temp_list)
                                        else:
                                            content = content[0]
                                        #print content
                                        if c['subContentRegex'] != None:
                                            pContent = self.get_list_by_regex(content, c['subContentRegex'], re.M|re.S)#re.compile(, re.M|re.S).findAll(content[0])
                                            if pContent != None:
                                                content = ("\n").join(pContent)
                                        #url = self.topList[self.mark][config_temp['topUrlId']]
                                       
                                        title = line[c['topTitleId']].decode(c["chart_set"])
                                        print title
                                        #print "-------1-------"
                                        content = re.subn(re.compile("<[^>]*>" , re.M|re.S),"",content)
                                        #print "-------2-------"
                                        content = re.subn(re.compile("&nbsp;" , re.M|re.S),"",content[0])
                                        #print "-------3-------"
                                        content =content[0]
                                        #print "-------4-------"
                                        avi_id += 1
                                        #print target_url,title,content
                                        if subContentListId == -1:
                                            
                                            try:
                                                
                                                self.sql.insert_new( target_url
                                                                 ,title
                                                                  , content)
                                            except Exception , e:
                                                print e
                                                self.sql.insert_new( target_url
                                                                 ,""
                                                                  , "")
                                                
                                        else:
                                            content = content.decode(c["chart_set"])
                                            self.sql.insert_new( target_url
                                                             ,title
                                                                  , "")
                                        #print "--------5------"
                                        #print content
                                        #添加
                                        
                                        if self.show != None:
                                            self.show.add_news((target_url,title,content))
                                        else:
                                            
                                            while True:
                                                if len(list) < 20:
                                                    
                                                        
                                                    list.append((target_url,title,content))
                                                    time.sleep(1)
                                                    break
                                                else:
                                                    time.sleep(1)
                                        
                                    #return "","",""#print self.mark,self.topList[self.mark][0]
                                 
                                        max_temp = c['max']
                                        print 'max=' + str(max_temp) + ",cur_id=" + str(avi_id)
                                        if max_temp > 0 and max_temp <= avi_id :
                                            print u'最大限定值，下一个配置'
        #                                    self.topList = None
        #                                    self.config_id = self.config_id + 1
        #                                    self.avi_id = 0
        #                                    self.mark = 0
                                            break
                
                                
                            except Exception , e:
                                print e
                                print u"异常：" + str(self.topList[self.mark][0])

                                
                               
        print u"程序结束"
                
               
    def getContent(self):
        while True:
            if len(self.news_list) > 0:
                return self.news_list.pop(0)
            else:
                time.sleep(1)
                print u"没有可用新闻"
        
if __name__ == '__main__':
    
    
    c = catch()
    while True:
        print c.getContent()
    
#    url = "http://news.sina.com.cn/"
#    topRegex = "<a href=\"(http://(ent|tech|games|slide)\.sina\.com\.cn/[^<>]+/\d+\.shtml)\" target=(\"|)_blank(\"|)>([^<>]+)</a>"
#    subRegex = "<!-- google_ad_section_start -->(.*)<!-- google_ad_section_end -->"
#    subContentRegex = "<p.*?>([^<>]+)</p>"
#    topUrlId=0
#    topTitleId=4
    
    
#    i=1
#    url ,title , content = c.getContent()
#    while content != None:
#        print "----------------------------------------------------------------------"
#        print i
#        i = i + 1
#        #print title.decode("gbk")
#        url , title , content = c.getContent()
#        print url , title , content
#        print "----------------------------------------------------------------------"
#    content = c.get_src("http://news.sina.com.cn/w/2010-12-20/142621674642.shtml").decode("gbk")
#    #print content
#    content =  c.get_list_by_regex(content, subRegex , re.M|re.S)
#    content = content[0]
#    
    
#    mark=0
#    for i in content:
#        print mark,i
#        mark+=1
#    test='''
#        123
#        <style >
#        .test{font:12px} 
#        </style>322
#    '''
#    print re.sub(re.compile("<(?P<name>style) >.*</\name>" , flags=20),"",test)
#    i=1
#    for href in c.get_list_by_regex(c.get_src(url), ""):
#        print i,href[0],href[4].decode("gbk")
#        content =  c.get_list_by_regex(c.get_src(href[0]) , "" , re.M|re.S)
#        if content != None and len(content) > 0:
#            content = re.subn(re.compile("<(a|style|script) .*?>.*</\0>" , re.M|re.S), "", content[0])
#            content = re.subn(re.compile("<.*?>" , re.M|re.S), "", content[0])
#            print content[0].decode("gbk")
#        else:
#            print u"内容为空"
#        i=i+1
    
